{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\numpy\\_distributor_init.py:30: UserWarning: loaded more than 1 DLL from .libs:\n",
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\numpy\\.libs\\libopenblas.FB5AE2TYXYH2IJRDKGDGQ3XBKLKTF43H.gfortran-win_amd64.dll\n",
      "d:\\Miniconda\\envs\\geo\\lib\\site-packages\\numpy\\.libs\\libopenblas64__v0.3.21-gcc_10_3_0.dll\n",
      "  warnings.warn(\"loaded more than 1 DLL from .libs:\"\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from peft import PeftModel\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = '/data/datasets/alpaca_data_zh/alpaca_gpt4_data_zh.json'\n",
    "pretrain_model_dir = \"/data/models/huggingface/bloom-1b4-zh\"\n",
    "save_dir = '/data/logs/conversation_rebot_for_1b4_lora_tuning'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载基础模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = AutoModelForCausalLM.from_pretrained(pretrain_model_dir)\n",
    "tokenizer = AutoTokenizer.from_pretrained(pretrain_model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载Lora模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Could not find the bitsandbytes CUDA binary at WindowsPath('d:/Miniconda/envs/geo/lib/site-packages/bitsandbytes/libbitsandbytes_cuda116.dll')\n",
      "The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers, 8-bit multiplication, and GPU quantization are unavailable.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PeftModelForCausalLM(\n",
       "  (base_model): LoraModel(\n",
       "    (model): BloomForCausalLM(\n",
       "      (transformer): BloomModel(\n",
       "        (word_embeddings): ModulesToSaveWrapper(\n",
       "          (original_module): Embedding(46145, 2048)\n",
       "          (modules_to_save): ModuleDict(\n",
       "            (default): Embedding(46145, 2048)\n",
       "          )\n",
       "        )\n",
       "        (word_embeddings_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (h): ModuleList(\n",
       "          (0): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (1): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (2): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (3): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (4): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (5): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (6): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (7): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (8): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (9): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (10): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (11): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (12): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (13): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (14): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (15): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (16): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (17): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (18): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (19): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (20): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (21): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (22): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (23): BloomBlock(\n",
       "            (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (self_attention): BloomAttention(\n",
       "              (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "              (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "              (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "            )\n",
       "            (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "            (mlp): BloomMLP(\n",
       "              (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "              (gelu_impl): BloomGelu()\n",
       "              (dense_4h_to_h): lora.Linear(\n",
       "                (base_layer): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Identity()\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=8192, out_features=8, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=8, out_features=2048, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (lora_magnitude_vector): ModuleDict()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (ln_f): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "      (lm_head): Linear(in_features=2048, out_features=46145, bias=False)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "p_model = PeftModel.from_pretrained(model, model_id=os.path.join(save_dir,\"checkpoint-1220/\"))\n",
    "p_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Human: 考试有哪些技巧？\\n\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: '"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ipt = tokenizer(\"Human: {}\\n{}\".format(\"考试有哪些技巧？\", \"\").strip() + \"\\n\\nAssistant: \", return_tensors=\"pt\")\n",
    "tokenizer.decode(p_model.generate(**ipt, max_length=256, do_sample=False)[0], skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型合并"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BloomForCausalLM(\n",
       "  (transformer): BloomModel(\n",
       "    (word_embeddings): Embedding(46145, 2048)\n",
       "    (word_embeddings_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "    (h): ModuleList(\n",
       "      (0): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (2): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (3): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (4): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (5): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (6): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (7): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (8): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (9): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (10): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (11): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (12): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (13): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (14): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (15): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (16): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (17): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (18): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (19): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (20): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (21): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (22): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "      (23): BloomBlock(\n",
       "        (input_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (self_attention): BloomAttention(\n",
       "          (query_key_value): Linear(in_features=2048, out_features=6144, bias=True)\n",
       "          (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
       "          (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (post_attention_layernorm): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): BloomMLP(\n",
       "          (dense_h_to_4h): Linear(in_features=2048, out_features=8192, bias=True)\n",
       "          (gelu_impl): BloomGelu()\n",
       "          (dense_4h_to_h): Linear(in_features=8192, out_features=2048, bias=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (ln_f): LayerNorm((2048,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=2048, out_features=46145, bias=False)\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge_model = p_model.merge_and_unload()\n",
    "merge_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Human: 考试有哪些技巧？\\n\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: 考试有哪些技巧？\\nAssistant: '"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ipt = tokenizer(\"Human: {}\\n{}\".format(\"考试有哪些技巧？\", \"\").strip() + \"\\n\\nAssistant: \", return_tensors=\"pt\")\n",
    "tokenizer.decode(merge_model.generate(**ipt, max_length=256, do_sample=False)[0], skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 完整模型保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m在当前单元格或上一个单元格中执行代码时 Kernel 崩溃。\n",
      "\u001b[1;31m请查看单元格中的代码，以确定故障的可能原因。\n",
      "\u001b[1;31m单击<a href='https://aka.ms/vscodeJupyterKernelCrash'>此处</a>了解详细信息。\n",
      "\u001b[1;31m有关更多详细信息，请查看 Jupyter <a href='command:jupyter.viewOutput'>log</a>。"
     ]
    }
   ],
   "source": [
    "merge_model.save_pretrained(os.path.join(save_dir,\"lora_model\"))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformers",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
